use maximum chunk size to decide the size of a bulk upload request

Signed-off-by: Matthieu Gallien <matthieu.gallien@nextcloud.com>

diff --git a/src/libsync/bulkpropagatorjob.cpp b/src/libsync/bulkpropagatorjob.cpp
index 9513b171165c3d6ae12b766eb0fcd6f29701bbe5..c03a82324b10a0440028b0120f53364c871ce9db 100644 (file)
--- a/src/libsync/bulkpropagatorjob.cpp
+++ b/src/libsync/bulkpropagatorjob.cpp
@@ -58,7 +58,6 @@ QByteArray getHeaderFromJsonReply(const QJsonObject &reply, const QByteArray &he
     return reply.value(headerName).toString().toLatin1();
 }
 
-constexpr auto batchSize = 100;
 constexpr auto parallelJobsMaximumCount = 1;
 
 }
@@ -70,10 +69,10 @@ Q_LOGGING_CATEGORY(lcBulkPropagatorJob, "nextcloud.sync.propagator.bulkupload",
 BulkPropagatorJob::BulkPropagatorJob(OwncloudPropagator *propagator, const std::deque<SyncFileItemPtr> &items)
     : PropagatorJob(propagator)
     , _items(items)
-    , _currentBatchSize(batchSize)
+    , _currentBatchSize(_items.size())
 {
-    _filesToUpload.reserve(batchSize);
-    _pendingChecksumFiles.reserve(batchSize);
+    _filesToUpload.reserve(_items.size());
+    _pendingChecksumFiles.reserve(_items.size());
 }
 
 bool BulkPropagatorJob::scheduleSelfOrChild()
@@ -84,11 +83,15 @@ bool BulkPropagatorJob::scheduleSelfOrChild()
 
     _state = Running;
 
-    for(auto i = 0; i < _currentBatchSize && !_items.empty(); ++i) {
+    qCDebug(lcBulkPropagatorJob()) << "max chunk size" << PropagatorJob::propagator()->syncOptions().maxChunkSize();
+
+    for(auto batchDataSize = 0; batchDataSize <= PropagatorJob::propagator()->syncOptions().maxChunkSize() && !_items.empty(); ) {
         const auto currentItem = _items.front();
         _items.pop_front();
         _pendingChecksumFiles.insert(currentItem->_file);
 
+        batchDataSize += currentItem->_size;
+
         QMetaObject::invokeMethod(this, [this, currentItem] {
             UploadFileInfo fileToUpload;
             fileToUpload._file = currentItem->_file;
@@ -117,7 +120,7 @@ bool BulkPropagatorJob::handleBatchSize()
     }
 
     // change batch size before trying it again
-    const auto halfBatchSize = batchSize / 2;
+    const auto halfBatchSize = static_cast<int>(_items.size() / 2);
 
     // we already tried to upload with half of the batch size
     if(_currentBatchSize == halfBatchSize) {

diff --git a/src/libsync/configfile.cpp b/src/libsync/configfile.cpp
index 1efd01e5da355610138ca32bc0d81009222658d5..8f875cc4d0f887265d31c8c153c993ed95e4a4ed 100644 (file)
--- a/src/libsync/configfile.cpp
+++ b/src/libsync/configfile.cpp
@@ -270,13 +270,13 @@ qint64 ConfigFile::chunkSize() const
 qint64 ConfigFile::maxChunkSize() const
 {
     QSettings settings(configFile(), QSettings::IniFormat);
-    return settings.value(QLatin1String(maxChunkSizeC), 5LL * 1000LL * 1000LL * 1000LL).toLongLong(); // default to 5000 MB
+    return settings.value(QLatin1String(maxChunkSizeC), 100LL * 1024LL * 1024LL).toLongLong(); // default to 100 MiB
 }
 
 qint64 ConfigFile::minChunkSize() const
 {
     QSettings settings(configFile(), QSettings::IniFormat);
-    return settings.value(QLatin1String(minChunkSizeC), 5LL * 1000LL * 1000LL).toLongLong(); // default to 5 MB
+    return settings.value(QLatin1String(minChunkSizeC), 5LL * 1024LL * 1024LL).toLongLong(); // default to 5 MiB
 }
 
 chrono::milliseconds ConfigFile::targetChunkUploadDuration() const

diff --git a/test/testsyncengine.cpp b/test/testsyncengine.cpp
index 26daf2c5fe63549aabfc7db70113964cf9971c78..21ca008f92582f17cd01c673e50e446500d893cc 100644 (file)
--- a/test/testsyncengine.cpp
+++ b/test/testsyncengine.cpp
@@ -1208,7 +1208,7 @@ private slots:
 
         QVERIFY(fakeFolder.syncOnce());
         QCOMPARE(nPUT, 0);
-        QCOMPARE(nPOST, 2);
+        QCOMPARE(nPOST, 1);
         nPUT = 0;
         nPOST = 0;
 
@@ -1219,7 +1219,7 @@ private slots:
 
         QVERIFY(!fakeFolder.syncOnce());
         QCOMPARE(nPUT, 120);
-        QCOMPARE(nPOST, 2);
+        QCOMPARE(nPOST, 1);
         nPUT = 0;
         nPOST = 0;